const path = require('path');
const fs = require('fs');
const http = require('http');

const colors = require('colors');
const mongoose = require('mongoose');
const cheerio = require('cheerio');
const request = require('request');
const iconv = require('iconv-lite');
const _ = require('underscore');

//const
const config = require('./config.json');
console.log('config:\n', config);
const listUrl='http://list.jd.com/list.html';
const bookUrl='http://item.jd.com/';

//Function
const utils = require('./modules/utils');

//models
const Book = require('./models/book');
const Comment = require('./models/comment');
const Record = require('./models/record');

colors.setTheme({
    info: 'cyan',
    log: 'white',
    error: 'red',
    warn: 'yellow'
});

const crawler = (function () {
    let self,curr,_canContinue;

    return {
        init: async function () {
            self = this;
            mongoose.connect(config.db);
            mongoose.Promise = global.Promise = require('bluebird');
            _canContinue=false;
            await self.createRecord();
            await self.start();
        },
        createRecord: function () {
            return new Promise((resolve, reject) => {
                Promise.all(
                    config.cats.map(cat =>Record.update({cat: cat}, {$setOnInsert:{page:0,count:0}}, {upsert: true}).exec())
                ).then(records => {
                    resolve(records);
                }).catch(err => {
                    utils.error('create records', err);
                    reject(err);
                });
            });
        },
        start:async function () {
            let records =await self.getRecord();
            curr=records.find(r=>!r.finish);
            if(!curr){
                utils.progress('全部分类抓取【完成】');
                return process.exit();
            }
            curr.page++;

            //列表
            utils.progress('开始抓取分类\n',curr);
            utils.progress(`开始抓取[${curr.cat.cn}]第${curr.page}页图书列表`);
            let html=await self.getCatPage();
            utils.progress(`抓取[${curr.cat.cn}]第${curr.page}页图书列表【完成】`);

            let parseCatPageRet=self.parseCatPage(html);
            utils.progress(`[${curr.cat.cn}]第${curr.page}页图书列表页共解析到[${parseCatPageRet.books.length}]本图书`);

            if(!self.canContinue(parseCatPageRet)){
                utils.progress('当前分类全部抓取完成',curr);
                await self.updateRecord(0);
                return self.start();
            }

            utils.progress(`开始抓取[${curr.cat.cn}]第${curr.page}页全部图书`);

            //详情
            let {books}=parseCatPageRet;
            let bookPages=await self.getBooks(books);
            let viewList=self.parseBookPages(books,bookPages);
            books=self.merge(books,viewList);
            if(!books.length){
                await self.updateRecord(0);
                return self.start();
            }
            utils.progress(`共正确解析到[${books.length}]本图书`);

            //价格
            let prices=await self.getPrice(books);
            books=self.merge(books,prices);
            if(!books.length){
                await self.updateRecord(0);
                return self.start();
            }
            utils.progress(`共获取到[${books.length}]本图书价格`);

            //Detail
            let details=await self.getDetail(books);
            books=self.merge(books,details);
            if(!books.length){
                await self.updateRecord(0);
                return self.start();
            }
            utils.progress(`共获取到[${books.length}]本图书详情`);

            //Comment
            let comments=await self.getComments(books);
            books=self.merge(books,comments);
            if(!books.length){
                await self.updateRecord(0);
                return self.start();
            }
            utils.progress(`共获取到[${books.length}]本图书评论`);


            //Save
            utils.progress(`开始保存[${curr.cat.cn}]第[${curr.page}]页全部[${books.length}]本图书原始数据`);
            await self.saveBooks(books);
            utils.progress(`保存[${curr.cat.cn}]第[${curr.page}]页全部[${books.length}]本图书原始数据【完成】`);

            //Download image
            utils.progress(`开始下载[${curr.cat.cn}]第[${curr.page}]页全部[${books.length}]本图书图片`);
            let downloadImgRets=await self.downloadAllImgs(books);
            utils.progress(`下载[${curr.cat.cn}]第[${curr.page}]页全部[${books.length}]本图书图片【完成】`);

            //Update
            utils.progress(`开始更新[${curr.cat.cn}]第${curr.page}页全部图书图片名称`);
            await self.updateImgsName(downloadImgRets);
            utils.progress(`更新[${curr.cat.cn}]第${curr.page}页全部[${books.length}]本图书图片名称【完成】`);
            await self.updateRecord(books.length);

            utils.progress('Finish\n',curr);

            //setTimeout(self.start,Math.random()*1000*5|0);
        },
        merge: function (books,list) {
            //过滤未正确抓取的图书
            list=list.filter(item=>item);
            return books.filter(book=>{
                let sku=book.sku,item=list.find(item=>sku===item.sku);
                if(!item)return false;
                Object.assign(book,item);
                return true;
            });
        },
        canContinue: function (ret) {
            _canContinue=ret.page.curr<ret.page.total && curr.count<=20;
            return _canContinue && ret.books.length;
        },
        getRecord: function () {
            return Record.find({}).exec();
        },
        getCatPage:async function () {
            return utils.getPage(listUrl,{cat:curr.cat.id,page:curr.page}).catch(err=>{
                utils.error('getPage error',err);
                process.exit(1);
            });
        },
        saveCatPage:function (filePath,data) {
            utils.writeFile(filePath,data).catch(err=>{
                self.warn('saveCatPage',err);
            });
        },
        parseCatPage: function (html) {
            let $=cheerio.load(html);
            //page
            let $page=$('#J_topPage');
            let page={
                curr:Number($page.find('.fp-text b').text()) || curr.page,//赋值数据库保存的数据
                total:Number($page.find('.fp-text i').text()) || curr.page+1//赋值数据库保存的数据并+1，使后面验证通过，继续抓取页面
            };
            //books
            let $list=$('#plist>ul>li'),books=[];
            for(let i=0;i<$list.length;i++){
                let $li= $list.eq(i);
                let sku=parseInt($li.find('.j-sku-item').data('sku'));
                let name=$li.find('.p-name em').text();
                let book={sku,name,cat:curr.cat};
                if(sku&&name){
                    books.push(book);
                }
            }
            return {page,books};
        },
        getBooks:function (books) {
            return Promise.all(
                books.map(book=>utils.getGBKPage(bookUrl+'/'+book.sku+'.html'))
            ).catch(err=>{
                utils.error('getBooks',err);
            });
        },
        parseBookPages:function (books,bookPages) {
            let viewList=[];
            books.forEach((book,i)=>{
                let $ = cheerio.load(bookPages[i]);
                let $preview=$('#preview'),$thumb = $preview.find('.spec-items img'),authorHtml=$('#p-author').html();
                if(!($preview.length&&$thumb.length&&authorHtml))return;

                let thumbs = [], previews = [];
                for (let j = 0; j < $thumb.length; j++) {
                    let $img = $thumb.eq(j);
                    let thumb=$img.attr('src');
                    thumbs.push(thumb);
                    let preview=thumb.replace(/\/n\d+\//,'/n'+$img.data('img')+'/');
                    previews.push(preview);
                }

                let author=authorHtml.replace(/\s*<a[\s\S]*?>\s*/g,'<em>').replace(/\s*<\/a>\s*/g,'</em>').trim();

                //商品介绍
                let paramStr = $('#parameter2').text().trim().replace(/\s+/g, '--');
                //console.log('paramStr:',paramStr);
                let paramReg = {//商品介绍
                    publishing: /出版社：--(.+?)--/,//出版社
                    isbn: /--ISBN：(\d+)--/,//ISBN
                    version: /--版次：(\d+)--/,//版次
                    code: /--商品编码：(\d+)--/,//商品编码
                    pack: /--包装：(.+?)--/,//包装
                    size: /--开本：(\d+)开--/,//开本
                    publishingTime: /--出版时间：(\d{4}-\d{2}-\d{2})--/,//出版时间
                    paper: /--用纸：(.+?)--/,//用纸
                    page: /--页数：(\d+)--///页数
                };

                let param = {},paramFlag=true;
                for (let name in paramReg) {
                    let m = paramStr.match(paramReg[name]);
                    //console.log('param reg:',book.sku,name,m);
                    if (!m || !m[1]) {
                        paramFlag=false;
                        break;
                    }
                    param[name] = m[1];
                }

                if(previews.length&&thumbs.length&&author&&paramFlag){
                    viewList.push({
                        sku:book.sku,
                        previews,
                        thumbs,
                        author,
                        param
                    });
                }
            });

            return viewList;
        },
        getPrice:async function (books) {
            return Promise.all(
                books.map(book=>new Promise((resolve)=>{
                    let uri='http://p.3.cn/prices/mgets?type=1&skuIds=J_'+book.sku;
                    request(uri,(err,res,body)=>{
                        if(err){
                            utils.warn('抓取价格失败',{book,uri,err});
                            return resolve(null);
                        }
                        let price;
                        try{
                            price=JSON.parse(body);
                        }catch(err){
                            if(err){
                                utils.warn('解析价格失败',{book,uri,err});
                                return resolve(null);
                            }
                        }

                        if(!price || !price[0] || !price[0].p || !price[0].m){
                            utils.warn('字段价格不存在',{book,uri});
                            return  resolve(null);
                        }
                        let p=price[0].p,m=price[0].m;
                        resolve({
                            sku:book.sku,
                            price:p,
                            market:m
                        });
                    });
                }))
            );
        },
        getDetail: function (books) {
            return Promise.all(
                books.map(book=>new Promise((resolve)=>{
                    http.get('http://d.3.cn/desc/'+book.sku+'?cdn=2&callback=showdesc', (res)=>{
                        if(200!=res.statusCode){
                            utils.warn('抓取Detail失败',view);
                            return resolve(null);
                        }
                        let chunks = [];
                        res.on('data', (chunk)=>{
                            chunks.push(chunk);
                        });
                        res.on('end', ()=>{
                            // 将二进制数据解码成 gb2312 编码数据
                            let txt=iconv.decode(Buffer.concat(chunks), 'gb2312'),detail=self.parseDetail(txt);
                            if(!detail)return resolve(null);

                            detail.sku=book.sku;
                            resolve(detail);
                        });
                    }).on('error',()=>{
                        utils.warn('抓取Detail失败',view);
                        return resolve(null);
                    });
                }))
            )
        },
        parseDetail: function (txt) {
            let con=txt.match(/\w+\(\{.+?"content":"([\s\S]+?)"\}\)/);
            if(!con){
                return null;
            }
            con=con[1];
            con=con.replace(/\\n/g,'').replace(/\\/g,'');

            let $=cheerio.load(con);
            let bjtj=$('#detail-tag-id-2').find('.book-detail-content p').html();//编辑推荐
            //console.log('bjtj:',bjtj);
            let nrjj=$('#detail-tag-id-3').find('.book-detail-content').html();//内容简介
            //console.log('nrjj:',nrjj);
            let zzjj=$('#detail-tag-id-4').find('.book-detail-content').html();//作者简介
            //console.log('zzjj:',zzjj);
            let jcsp=$('#detail-tag-id-5').find('.book-detail-content').html();//精彩书评
            let ml=$('#detail-tag-id-6').find('.book-detail-content').html();//目录
            let jcsz=$('#detail-tag-id-7').find('.book-detail-content').html();//精彩书摘
            let qy=$('#detail-tag-id-8').find('.book-detail-content').html();////前言/序言

            return {
                bjtj,nrjj,zzjj,jcsp,ml,jcsz,qy
            }
        },
        getComments:function(books){
            return Promise.all(
                books.map(book=>new Promise((resolve,reject)=>{
                    let uri=`http://sclub.jd.com/comment/productPageComments.action?productId=${book.sku}&score=0&sortType=3&page=0&pageSize=100`;
                    http.get(uri, (res)=>{
                        let chunks = [];
                        res.on('data', (chunk)=>{
                            chunks.push(chunk);
                        });
                        res.on('end', ()=>{
                            // 将二进制数据解码成 gb2312 编码数据
                            let gbk = iconv.decode(Buffer.concat(chunks), 'gb2312');
                            let comment=self.parseComments(book,gbk);
                            resolve({sku:book.sku,comment});
                        });
                        res.on('error',err=>{
                            utils.warn('getComments',err);
                            resolve(null);
                        });
                    });
                }))
            )
        },
        parseComments: function (book,txt) {
            let json;
            try{
                json=JSON.parse(txt);
            }catch(e){
                return null;
            }
            //console.log('\ncomment json:',json);
            //console.log('\ncomment json.comments.length:',json.comments.length);
            let summary=json.productCommentSummary;
            let c={
                sku:summary.skuId,//sku

                commentCount:summary.commentCount,//总评论数
                commentCountStr:summary.commentCountStr,

                goodCount:summary.goodCount,//好评
                goodRate:summary.goodRate,//好评率
                goodCountStr:summary.goodCountStr,

                generalCount:summary.generalCount,//中评
                generalRate:summary.generalRate,
                generalCountStr:summary.generalCountStr,

                poorCount:summary.poorCount,//差评
                poorRate:summary.poorRate,
                poorCountStr:summary.poorCountStr

            };
            let total=Number(summary.commentCount);
            let length=json.comments.length;
            c.total=total || length || 0;

            let cs=json.comments.map((c)=>{
                return {
                    score:c.score,
                    content:c.content,
                    time:c.creationTime,
                    images:c.images?c.images.map((img)=>{
                        return img.imgUrl;
                    }):[],
                    nick:c.nickname
                }
            });
            c.comments=cs;

            //检查数据完整性
            let fail=['sku','commentCount','goodCount','generalCount'].filter((field)=>{
                return undefined===c[field];
            });

            if(fail.length){
                utils.warn('评价字段'+fail+'解析失败',book.sku);
                return null;
            }
            return c;
        },
        saveBooks: function (books) {
            let comments=books.map(book=>book.comment);

            //生成 mongoDB _id
            let newBooks=books.map(book=>new Book(book));
            let newComments=comments.map(comment=>new Comment(comment));

            //模拟数据+添加 _id
            newBooks.forEach((book,i)=>{
                let comment=newComments[i],firstComemntTime=new Date((_.last(comment.comments)||{time:new Date()}).time).getTime();
                let publishingTime=new Date(book.param.publishingTime).getTime();
                book.putaway=_.random(publishingTime,firstComemntTime);/*上架时间，取第一个评论和出版时间的中间值*/
                book.sales=(Math.random()*0.9+1.1)*comment.commentCount|0;
                book.comment_id=comment._id;
                comment.book_id=book._id;
            });

            return Promise.all([
                    Promise.all(newBooks.map(book=>new Promise((resolve, reject)=> {
                        book.save((err, ret)=> {
                            if (err)return utils.error('保存book', {book: book, error: err}), reject(err);
                            resolve(ret);
                        });
                    }))),
                    Promise.all(newComments.map(comment=>new Promise((resolve)=> {
                        comment.save((err, ret)=> {
                            if (err)return utils.error('保存comment', {comment: comment, error: err}), reject(err);
                            resolve(ret);
                        });
                    })))
                ]
            ).catch(err=>{
                utils.error('批量保存books&comments', err);
            });
        },
        downloadImg:async function(data){
            let name=path.basename(data.uri),size=data.size||'',fileName=`${data.sku}_${data.idx}_${size}_${name}`;
            let filePath=path.join(config.downloadDir,data.dir,fileName);
            let ac=await utils.access(filePath);

            if(ac)return {ret:'YET',data:data,fileName};

            return new Promise(resolve=>{
                //console.log(data,fileName);
                request.get('http:'+data.uri,(err)=>{
                    if(err){
                        utils.warn('下载图片失败',data);
                        return resolve({error:err,data:data,fileName});
                    }
                    resolve({"ret":"OK","data":data,"fileName":fileName});
                }).pipe(fs.createWriteStream(filePath));
            });

        },
        downloadAllImgs:function (books) {
            return Promise.all([
                self.downloadImgs(books,'previews'),
                self.downloadImgs(books,'thumbs'),
                self.downloadCommentImgs(books,'76x76'),
                self.downloadCommentImgs(books,'616x405')
            ]).catch(err=> {
                utils.error('下载图片', err);
            });
        },
        downloadImgs:function(books,field){
            utils.progress(`开始下载${field}图片`);
            return Promise.all(
                books.map(book=>Promise.all(
                    book[field].map((uri,i)=>self.downloadImg({uri:uri,sku:book.sku,idx:i,dir:field,field:field}))
                ))
            ).catch(err=> {
                utils.error(`下载${field}图片`, err);
            });
        },
        downloadCommentImgs: function (books,size) {
            utils.progress(`开始下载comments ${size} 大小的图片`);
            return Promise.all(
                books.map(book=>Promise.all(
                    book.comment.comments.map((item,i)=>Promise.all(
                        item.images.map(uri=>{
                            uri=uri.replace('128x96',size);
                            return self.downloadImg({uri:uri,idx:i,sku:book.sku,dir:'comments',field:'76x76'==size?'smalls':'bigs'});
                        })
                    ))
                ))
            ).catch(err=> {
                utils.error(`下载comments size:${size}图片`, err);
            });
        },
        updateImgsName: function (rets) {
            //展开为一维数组
            rets=rets.map(list=>utils.flatten(list));

            //过滤抓取错误的图片
            rets=rets.map(list=>{
                return list.filter(item=>!item.error);
            });

            let [books,comments]=[[...rets[0],...rets[1]],[...rets[2],...rets[3]]];

            let bookUpdates=[];
            for(let item of books){
                let sku=item.data.sku,field=item.data.field,fileName=item.fileName;
                let book=bookUpdates.find(item=>item.sku==sku&&item.field==field);
                if(book){
                    book.data.push(fileName);
                }else{
                    bookUpdates.push({
                        sku,
                        field,
                        data:[fileName]
                    });
                }
            }

            let commentUpdates=[];
            for(let item of comments){
                let sku=item.data.sku,field=item.data.field,fileName=item.fileName,idx=item.data.idx;
                let comment=commentUpdates.find(item=>item.sku==sku&&item.idx==idx&&field==item.field);
                if(comment){
                    comment.data.push(fileName);
                }else{
                    commentUpdates.push({
                        sku,
                        idx,
                        field,
                        data:[fileName]
                    });
                }
            }

            return Promise.all([
                Promise.all(
                    bookUpdates.map(
                        book=>Book.update({sku:book.sku},{[book.field]:book.data}).exec()
                    )
                ),
                Promise.all(
                    commentUpdates.map(
                        comment=>Comment.update({sku:comment.sku},{[`comments.${comment.idx}.${comment.field}`]:comment.data})
                    )
                )
            ]).catch(err=>{
                utils.error('Update image name',err);
            });

        },
        updateRecord: function (count) {
            curr.finish=!_canContinue;
            curr.count+=count;
            return Record.update({'cat.id':curr.cat.id},{finish:!_canContinue,$inc:{page:1,count:count}}).exec();
        }

    }
})();
crawler.init();